<?php
/* Do NOT delete this comment */
/* 不要删除这段注释 */
namespace App\Noxinfluencer\Controllers;
require_once '../../BaseController.php';
require_once '../../../vendor/autoload.php';

use App\BaseController;
use Medoo\Medoo;
use phpspider\core\log;
use phpspider\core\phpspider;
use phpspider\core\requests;
use phpspider\core\selector;

class UserSpiderController extends BaseController
{
    const SOURCE_ID = 1;                    //爬虫爬取数据来源       1 夜神

    const TABLE = 'kol_user';

    const LOG_FILE = "/data/spider/noxinfluencer/user_spide.log";      //爬虫日志

    const PAGE_SIZE = 300;            //每个国家爬取页数

    const NOT_SEARCH_COUNTRY = [];

    const SEARCH_COUNTRY = array('us','in','ca', 'fi', 'se', 'no', 'is', 'dk', 'lt', 'by', 'ru', 'ua', 'pl', 'cz', 'hu', 'de', 'at', 'ch', 'uk', 'ie', 'nl', 'be', 'lu', 'fr', 'mc', 'ro', 'rs', 'it', 'es', 'pt', 'au', 'nz', 'tr', 'bh', 'ir', 'iq', 'il', 'kw', 'ae', 'ar', 'br', 'co', 'mx', 'jp', 'kr', 'my', 'sg', 'vn', 'th');

    public function __construct()
    {

    }


    public function startSpider()
    {
        $pageSize = self::PAGE_SIZE;

        $all_country = $this->medooDb()->select("kol_country", ['country_simple_name', 'country_china_name'], ['country_simple_name' => self::SEARCH_COUNTRY]);

        $not_search_country = self::NOT_SEARCH_COUNTRY;


        $spider = $this->spiderObj();

        $this->spiderOnExtractField($spider,$this->getAllLanguage());

        $this->spiderOnFetchUrl($spider);

        $this->spiderOnStart($spider,$all_country, $not_search_country, $pageSize);


        $spider->start();
    }



    /**
     * 获取所以国家
     * @return array
     */
    public function getAllCountry():array
    {
        $all_country = [];
        $country = $this->medooDb()->select("kol_country", ['country_simple_name', 'country_china_name']);
        foreach ($country as $key=>$value){
            $all_country[strtolower($value['country_simple_name'])] = $value['country_china_name'];
        }
        return $all_country;
    }

    /**
     * 获取所有语言  language_name => id
     * @return array
     */
    public function getAllLanguage():array
    {
        $all_data = [];
        $data = $this->medooDb()->select("kol_language", ['id', 'category_name']);
        foreach ($data as $key=>$value){
            $all_data[$value['category_name']] = $value['id'];
        }
        return $all_data;
    }


    /**
     * 获取爬取数据语言对应的语言表id
     * @param $spider_language_data
     * @return int
     */
    public function getLanguageId($spider_language_data):int
    {
        $medooDb = $this->medooDb();

        $id = $medooDb->get('kol_language','id',['category_name'=>$spider_language_data]);
        if($id)
            return $id;

        $medooDb->insert('kol_language',['category_name'=>$spider_language_data,'create_time'=>date('Y-m-d H:i:s',time()),'update_time'=>date('Y-m-d H:i:s',time())]);
        return $medooDb->id();
    }

    /**
     * 获取爬取数据 国家对应的国家表id
     * @param $spider_data
     * @return int
     */
    public function getCountryId($spider_data):int
    {
        $medooDb = $this->medooDb();

        $id = $medooDb->get('kol_country','id',['country_china_name'=>$spider_data]);
        if($id)
            return $id;

        $medooDb->insert('kol_country',['country_china_name'=>$spider_data,'create_time'=>date('Y-m-d H:i:s',time()),'update_time'=>date('Y-m-d H:i:s',time())]);
        return $medooDb->id();
    }



    public function formatNum($data)
    {
        if(strpos($data,'万元')){
            return trim(str_replace(['万元'], [''], $data)) * 10000;
        }

        if(strpos($data,'万')){
            return trim(str_replace(['万'], [''], $data)) * 10000;
        }

        if(strpos($data,'亿')){
            return trim(str_replace(['亿'], [''], $data)) * 100000000;
        }

        return $data;
    }



    public function spiderObj(){
        return new phpspider($this->spiderConfig());
    }





    public function spiderOnFetchUrl(phpspider $spider){
        $spider->on_fetch_url = function ($url, $spider) {
            if (strpos($url, "mailto") !== false) {
                return false;
            }
            return $url;
        };
    }



    public function spiderOnStart(phpspider $spider,$all_country, $not_search_country, $pageSize){
        $spider->on_start = function ($spider) use ($all_country, $not_search_country, $pageSize) {
            foreach ($all_country as $key => $value) {
                $country_simple_name = strtolower($value['country_simple_name']);
                if (!in_array($country_simple_name, $not_search_country)) {
                    // 生成列表页URL入队列
                    for ($i = 1; $i <= $pageSize; $i++) {
                        $url = "https://cn.noxinfluencer.com/youtube/search/?isvideo=0&category=all&follower=0&area={$country_simple_name}&exposure=0&published=0&language=all&noxScoreLevel=0&pageNum={$i}";
                        $spider->add_scan_url($url);
                    }
                }
            }
        };
    }


    public function spiderOnExtractField(phpspider $spider,$language)
    {
        $spider->on_extract_field = function ($fieldname, $data, $page) use ($language) {
            if ($fieldname == 'channel_id') {
                if (strpos($data, "channel/") !== false) {
                    $channel = explode('channel/', $data);
                    return $channel[1];
                }
            }

            if ($fieldname == 'name') {
                return str_replace('&amp;', '&', $data);
            }

            if ($fieldname == 'fans_num') {
                return $this->formatNum($data);
            }

            if ($fieldname == 'video_look_num') {
                return $this->formatNum($data[1]);
            }

            if ($fieldname == 'video_avg_look_num') {
                return $this->formatNum($data[2]);
            }

            if ($fieldname == 'video_num') {
                return $this->formatNum($data);
            }

            if ($fieldname == 'country_id') {
                $country = trim(preg_replace("/(\s|\ \;| |\xc2\xa0)/", "", $data));
                return $this->getCountryId($country);
            }

            if ($fieldname == 'language_id') {
                $language = trim(preg_replace("/(\s|\ \;| |\xc2\xa0)/", "", $data));
                return $this->getLanguageId($language);
            }

            /*if ($fieldname == 'category_id') {
                return preg_replace("/(\s|\ \;| |\xc2\xa0)/", "", $data);
            }*/

            if ($fieldname == 'teamwork_fee') {
                return $this->formatNum($data);
            }

            if ($fieldname == 'video_prediction_play_num') {
                return $this->formatNum(trim(str_replace(['预估曝光量:  '], [''],$data)));
            }

            /*if ($fieldname == 'grade')
            {
                return str_replace(['<i class="kol-icon kol-icon-star-full"></i>','<i class="kol-icon kol-icon-star-empty"></i>','<i class="kol-icon kol-icon-star-half"></i>'],['','',''],$data[0]);
            }*/

            if ($fieldname == 'present_month_video_num') {
                return $data[1];
            }

            if ($fieldname == 'world_fans_sort') {
                if(empty($data) || !isset($data[0]) || $data[0]=='nullth'){
                    return 1000000000;
                }
                return str_replace([',','th','st'], ['', '',''], $data[0]);
            }

            if ($fieldname == 'country_fans_sort') {
                if(empty($data) || !isset($data[1]) || $data[1]=='nullth'){
                    return 1000000000;
                }
                return str_replace([',','th','st'], ['', '',''], $data[1]);
            }

            if ($fieldname == 'source_id'){
                return self::SOURCE_ID;
            }

            if ($fieldname == 'create_time') {
                return date('Y-m-d H:i:s', time());
            }

            if ($fieldname == 'update_time') {
                return date('Y-m-d H:i:s', time());
            }

            return $data;
        };
    }


    /**
     * 爬虫配置
     * @return array
     */
    public function spiderConfig():array
    {
        return array(
            'name' => 'noxinfluencer',
            'log_show' => false,
            'log_file' => self::LOG_FILE,
            //    'save_running_state' => true,
            'tasknum' => 1,
            'max_depth' => 1,
            //数据库配置
            'db_config' => self::spiderDb(),
            'queue_config' => self::spiderQueueDb(),
            'export' => array(
                'type' => 'db',
                'table' => self::TABLE,  // 如果数据表没有数据新增请检查表结构和字段名是否匹配
            ),
            //爬取的域名列表
            'domains' => array(
                'cn.noxinfluencer.com'
            ),
            //抓取的起点
            'scan_urls' => array(
                "https://cn.noxinfluencer.com/youtube/search/?isvideo=0&category=all&follower=0&area=us&exposure=0&published=0&language=all&noxScoreLevel=0&pageNum=1",
            ),
            //列表页实例
            'list_url_regexes' => array(
                "https://cn.noxinfluencer.com/youtube/search/?isvideo=0&category=all&follower=0&area=us&exposure=0&published=0&language=all&noxScoreLevel=0&pageNum=1",
            ),
            //内容页实例
            'content_url_regexes' => array(
                "https://cn.noxinfluencer.com/youtube/channel/[A-Za-z0-9]+",
            ),
            'max_try' => 5,

            'client_ip' => self::spiderClientIp(),

            'fields' => array(
                array(
                    'name' => "name",
                    'selector' => "//div[@class='title']/span",
                ),
                array(
                    'name' => "channel_id",
                    'selector' => "//div[@class='info-header']/div[@class='socials']/a[@class='icon-wrapper']/@href",
                    'required' => true,
                ),
                array(
                    'name' => "fans_num",
                    'selector' => "//div[@class='info-bottom']//div[@class='column']//div[@class='sub-title']//span[@class='strong']",
                ),
                array(
                    'name' => "video_look_num",
                    'selector' => "//div[@class='info-bottom']//span[@class='strong']",
                    'repeated' => true,
                ),
                array(
                    'name' => "video_avg_look_num",
                    'selector' => "//div[@class='info-bottom']//span[@class='strong']",
                    'repeated' => true,
                ),
                array(
                    'name' => "video_num",
                    'selector' => "//div[@class='info-bottom']//div[@class='column'][last()]//span[@class='strong']",
                ),
                array(
                    'name' => "country_id",
                    'selector' => "//div[@class='info-content']//span[@class='subtitle underline']",
                ),
                array(
                    'name' => "language_id",
                    'selector' => "//div[@class='info-content']//span[@class='subtitle'][last()]",
                ),
                array(
                    'name' => "source_head_img",
                    'selector' => "//div[@class='avatar-wrapper']/img/@src",
                ),
                /*array(
                    'name' => "category_id",
                    'selector' => "//div[@class='info-container']//div[@class='info-header']//div[@class='tags']//a",
                ),*/
                array(
                    'name' => "register_time",
                    'selector' => "//div[@class='info-content']/span[@class='title']/span[@class='subtitle']",
                ),
                array(
                    'name' => "video_prediction_play_num",
                    'selector' => "//div[@class='card-footer'][last()]//span[last()]",
                ),
                array(
                    'name' => "teamwork_fee",
                    'selector' => "//div[@class='card-content'][last()]//span[@class='title']",
                ),
                /*array(
                    'name' => "grade",
                    'selector' => "//div[@class='card-content']//span[@class='title']",
                    'required' => false,
                    'repeated' => true,
                ),*/
                array(
                    'name' => "present_month_video_num",
                    'selector' => "//div[@class='card-content']//span[@class='title']",
                    'repeated' => true,
                ),
                array(
                    'name' => "world_fans_sort",
                    'selector' => "//div[@class='card-content']//a[@class='underline title']",
                    'repeated' => true,
                ),
                array(
                    'name' => "country_fans_sort",
                    'selector' => "//div[@class='card-content']//a[@class='underline title']",
                    'repeated' => true,
                ),
                array(
                    'name' => "source_id",
                    'selector' => "",
                ),
                array(
                    'name' => "create_time",
                    'selector' => "",
                ),
                array(
                    'name' => "update_time",
                    'selector' => "",
                ),
            ),
        );
    }

}

(new UserSpiderController())->startSpider();





